{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "from puckworld_continuous import PuckWorldEnv\n",
    "from ddpg_agent import DDPGAgent\n",
    "from utils import learning_curve\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = PuckWorldEnv() \n",
    "agent = DDPGAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent.load_models(300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 1/200 [00:00<02:32,  1.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 0     156 steps,total reward:-62.45  \n",
      "episode:  0：loss critic:0.007, loss_actor:8.906\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  1%|          | 2/200 [01:12<2:00:06, 36.39s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 1    7208 steps,total reward:-4814.08\n",
      "episode:  1：loss critic:0.004, loss_actor:245.683\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  2%|▏         | 3/200 [01:28<1:36:39, 29.44s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 2    1496 steps,total reward:-810.58 \n",
      "episode:  2：loss critic:0.013, loss_actor:505.883\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  2%|▏         | 4/200 [01:37<1:19:25, 24.32s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 3     868 steps,total reward:-571.86 \n",
      "episode:  3：loss critic:0.019, loss_actor:597.236\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  2%|▎         | 5/200 [02:09<1:24:27, 25.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 4    3164 steps,total reward:-2073.45\n",
      "episode:  4：loss critic:0.024, loss_actor:761.890\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  3%|▎         | 6/200 [02:12<1:11:24, 22.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 5     248 steps,total reward:-92.32  \n",
      "episode:  5：loss critic:0.030, loss_actor:817.080\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  4%|▎         | 7/200 [02:12<1:01:01, 18.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 6      24 steps,total reward:-6.79   \n",
      "episode:  6：loss critic:0.027, loss_actor:787.771\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  4%|▍         | 8/200 [02:14<53:36, 16.75s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 7     121 steps,total reward:-31.15  \n",
      "episode:  7：loss critic:0.022, loss_actor:749.713\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  4%|▍         | 9/200 [02:16<48:12, 15.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 8     210 steps,total reward:-81.47  \n",
      "episode:  8：loss critic:0.019, loss_actor:703.743\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  5%|▌         | 10/200 [02:16<43:15, 13.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 9      31 steps,total reward:-11.27  \n",
      "episode:  9：loss critic:0.032, loss_actor:697.072\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|▌         | 12/200 [02:18<36:09, 11.54s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 10    159 steps,total reward:-51.68  \n",
      "episode: 10：loss critic:0.023, loss_actor:690.811\n",
      "episode 11     15 steps,total reward:-9.43   \n",
      "episode: 11：loss critic:0.027, loss_actor:687.595\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  6%|▋         | 13/200 [02:19<33:23, 10.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 12     70 steps,total reward:-21.67  \n",
      "episode: 12：loss critic:0.035, loss_actor:687.398\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  7%|▋         | 14/200 [02:22<31:29, 10.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 13    278 steps,total reward:-78.91  \n",
      "episode: 13：loss critic:0.033, loss_actor:673.102\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  8%|▊         | 15/200 [02:25<29:54,  9.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 14    322 steps,total reward:-79.08  \n",
      "episode: 14：loss critic:0.029, loss_actor:661.370\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  8%|▊         | 16/200 [02:26<28:01,  9.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 15     67 steps,total reward:-15.75  \n",
      "episode: 15：loss critic:0.031, loss_actor:635.529\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  8%|▊         | 17/200 [02:29<26:47,  8.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 16    289 steps,total reward:-112.65 \n",
      "episode: 16：loss critic:0.032, loss_actor:633.208\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  9%|▉         | 18/200 [02:29<25:13,  8.31s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 17     28 steps,total reward:-4.44   \n",
      "episode: 17：loss critic:0.021, loss_actor:630.060\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 10%|▉         | 19/200 [02:31<23:59,  7.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 18    134 steps,total reward:-52.15  \n",
      "episode: 18：loss critic:0.038, loss_actor:630.027\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 10%|█         | 20/200 [02:31<22:44,  7.58s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 19     41 steps,total reward:-16.71  \n",
      "episode: 19：loss critic:0.040, loss_actor:625.768\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11%|█         | 22/200 [02:32<20:30,  6.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 20     33 steps,total reward:-4.86   \n",
      "episode: 20：loss critic:0.043, loss_actor:601.030\n",
      "episode 21     15 steps,total reward:-1.67   \n",
      "episode: 21：loss critic:0.041, loss_actor:582.928\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 12%|█▏        | 23/200 [02:32<19:35,  6.64s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 22     56 steps,total reward:-19.94  \n",
      "episode: 22：loss critic:0.036, loss_actor:593.475\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 12%|█▏        | 24/200 [02:32<18:41,  6.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 23     20 steps,total reward:-0.57   \n",
      "episode: 23：loss critic:0.034, loss_actor:563.668\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 12%|█▎        | 25/200 [02:33<17:56,  6.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 24     85 steps,total reward:-43.10  \n",
      "episode: 24：loss critic:0.042, loss_actor:573.321\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 13%|█▎        | 26/200 [02:35<17:19,  5.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 25    137 steps,total reward:-29.38  \n",
      "episode: 25：loss critic:0.037, loss_actor:560.822\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|█▍        | 28/200 [02:35<15:58,  5.57s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 26     44 steps,total reward:-17.28  \n",
      "episode: 26：loss critic:0.039, loss_actor:564.362\n",
      "episode 27     17 steps,total reward:-2.63   \n",
      "episode: 27：loss critic:0.043, loss_actor:554.233\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 14%|█▍        | 29/200 [02:36<15:22,  5.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 28     48 steps,total reward:-16.72  \n",
      "episode: 28：loss critic:0.042, loss_actor:544.113\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 15%|█▌        | 30/200 [02:37<14:52,  5.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 29     93 steps,total reward:-14.15  \n",
      "episode: 29：loss critic:0.039, loss_actor:539.617\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 16%|█▌        | 31/200 [02:38<14:22,  5.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 30     54 steps,total reward:-15.09  \n",
      "episode: 30：loss critic:0.034, loss_actor:542.232\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 16%|█▌        | 32/200 [02:38<13:51,  4.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 31     25 steps,total reward:-7.80   \n",
      "episode: 31：loss critic:0.038, loss_actor:554.765\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 16%|█▋        | 33/200 [02:40<13:32,  4.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 32    203 steps,total reward:-81.72  \n",
      "episode: 32：loss critic:0.036, loss_actor:543.299\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 17%|█▋        | 34/200 [02:41<13:08,  4.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 33     91 steps,total reward:-22.17  \n",
      "episode: 33：loss critic:0.027, loss_actor:534.026\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 18%|█▊        | 35/200 [02:42<12:44,  4.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 34     49 steps,total reward:-8.63   \n",
      "episode: 34：loss critic:0.029, loss_actor:535.946\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 18%|█▊        | 36/200 [02:48<12:47,  4.68s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 35    548 steps,total reward:-198.12 \n",
      "episode: 35：loss critic:0.031, loss_actor:533.361\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 18%|█▊        | 37/200 [02:49<12:24,  4.57s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 36     47 steps,total reward:-18.60  \n",
      "episode: 36：loss critic:0.044, loss_actor:561.564\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 19%|█▉        | 38/200 [02:52<12:16,  4.55s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 37    355 steps,total reward:-131.28 \n",
      "episode: 37：loss critic:0.035, loss_actor:540.684\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 20%|█▉        | 39/200 [02:53<11:54,  4.44s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 38     24 steps,total reward:-5.77   \n",
      "episode: 38：loss critic:0.036, loss_actor:549.383\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 20%|██        | 40/200 [02:54<11:36,  4.35s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 39     92 steps,total reward:-32.53  \n",
      "episode: 39：loss critic:0.033, loss_actor:559.745\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|██        | 42/200 [02:54<10:57,  4.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 40     55 steps,total reward:-9.34   \n",
      "episode: 40：loss critic:0.028, loss_actor:547.559\n",
      "episode 41     15 steps,total reward:-1.40   \n",
      "episode: 41：loss critic:0.047, loss_actor:536.360\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 22%|██▏       | 43/200 [02:55<10:40,  4.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 42     41 steps,total reward:-18.43  \n",
      "episode: 42：loss critic:0.032, loss_actor:546.748\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 22%|██▏       | 44/200 [02:57<10:27,  4.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 43    164 steps,total reward:-51.10  \n",
      "episode: 43：loss critic:0.029, loss_actor:538.590\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|██▎       | 46/200 [02:57<09:54,  3.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 44     38 steps,total reward:-17.09  \n",
      "episode: 44：loss critic:0.030, loss_actor:554.435\n",
      "episode 45     15 steps,total reward:-1.51   \n",
      "episode: 45：loss critic:0.031, loss_actor:542.363\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 24%|██▎       | 47/200 [03:07<10:11,  4.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 46    955 steps,total reward:-372.68 \n",
      "episode: 46：loss critic:0.031, loss_actor:549.341\n",
      "episode 47      1 steps,total reward:0.02    \n",
      "episode: 47：loss critic:0.058, loss_actor:479.070\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 24%|██▍       | 49/200 [03:10<09:46,  3.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 48    243 steps,total reward:-63.74  \n",
      "episode: 48：loss critic:0.030, loss_actor:551.807\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 25%|██▌       | 50/200 [03:11<09:35,  3.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 49    149 steps,total reward:-38.23  \n",
      "episode: 49：loss critic:0.035, loss_actor:547.810\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 26%|██▌       | 51/200 [03:17<09:36,  3.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 50    504 steps,total reward:-184.97 \n",
      "episode: 50：loss critic:0.036, loss_actor:555.603\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 26%|██▌       | 52/200 [03:19<09:26,  3.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 51    171 steps,total reward:-35.95  \n",
      "episode: 51：loss critic:0.037, loss_actor:557.549\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 26%|██▋       | 53/200 [03:19<09:13,  3.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 52     44 steps,total reward:-10.70  \n",
      "episode: 52：loss critic:0.036, loss_actor:564.161\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 27%|██▋       | 54/200 [03:22<09:08,  3.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 53    306 steps,total reward:-112.76 \n",
      "episode: 53：loss critic:0.035, loss_actor:562.475\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 28%|██▊       | 55/200 [03:23<08:55,  3.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 54     51 steps,total reward:-8.58   \n",
      "episode: 54：loss critic:0.026, loss_actor:557.613\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 28%|██▊       | 56/200 [03:24<08:46,  3.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 55    144 steps,total reward:-45.05  \n",
      "episode: 55：loss critic:0.039, loss_actor:564.891\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 28%|██▊       | 57/200 [03:26<08:38,  3.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 56    184 steps,total reward:-66.23  \n",
      "episode: 56：loss critic:0.037, loss_actor:564.071\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 29%|██▉       | 58/200 [03:27<08:28,  3.58s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 57     80 steps,total reward:-19.27  \n",
      "episode: 57：loss critic:0.028, loss_actor:561.052\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 60/200 [03:28<08:05,  3.47s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 58     30 steps,total reward:-1.39   \n",
      "episode: 58：loss critic:0.029, loss_actor:552.594\n",
      "episode 59     17 steps,total reward:-1.37   \n",
      "episode: 59：loss critic:0.037, loss_actor:541.414\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 30%|███       | 61/200 [03:29<07:57,  3.43s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 60    135 steps,total reward:-27.42  \n",
      "episode: 60：loss critic:0.034, loss_actor:560.757\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 31%|███       | 62/200 [03:29<07:47,  3.39s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 61     32 steps,total reward:-5.24   \n",
      "episode: 61：loss critic:0.044, loss_actor:541.076\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 32%|███▏      | 63/200 [03:30<07:37,  3.34s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 62     35 steps,total reward:-14.04  \n",
      "episode: 62：loss critic:0.036, loss_actor:544.758\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 32%|███▏      | 64/200 [03:31<07:28,  3.30s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 63     79 steps,total reward:-24.96  \n",
      "episode: 63：loss critic:0.036, loss_actor:563.523\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 32%|███▎      | 65/200 [03:31<07:20,  3.26s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 64     79 steps,total reward:-19.96  \n",
      "episode: 64：loss critic:0.036, loss_actor:539.203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 33%|███▎      | 66/200 [03:41<07:30,  3.36s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 65    939 steps,total reward:-240.57 \n",
      "episode: 65：loss critic:0.040, loss_actor:553.538\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 34%|███▎      | 67/200 [03:42<07:21,  3.32s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 66     57 steps,total reward:-12.56  \n",
      "episode: 66：loss critic:0.036, loss_actor:552.578\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 34%|███▍      | 68/200 [03:44<07:16,  3.30s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 67    227 steps,total reward:-67.13  \n",
      "episode: 67：loss critic:0.044, loss_actor:561.405\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 34%|███▍      | 69/200 [03:45<07:07,  3.27s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 68     59 steps,total reward:-18.79  \n",
      "episode: 68：loss critic:0.029, loss_actor:571.581\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 35%|███▌      | 70/200 [03:47<07:03,  3.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 69    236 steps,total reward:-79.84  \n",
      "episode: 69：loss critic:0.044, loss_actor:563.243\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 36%|███▌      | 71/200 [03:49<06:57,  3.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 70    192 steps,total reward:-60.32  \n",
      "episode: 70：loss critic:0.040, loss_actor:568.141\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 36%|███▌      | 72/200 [03:50<06:48,  3.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 71     23 steps,total reward:-3.69   \n",
      "episode: 71：loss critic:0.057, loss_actor:557.517\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 36%|███▋      | 73/200 [03:50<06:41,  3.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 72     60 steps,total reward:-14.64  \n",
      "episode: 72：loss critic:0.047, loss_actor:555.602\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 37%|███▋      | 74/200 [03:51<06:33,  3.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 73     41 steps,total reward:-11.19  \n",
      "episode: 73：loss critic:0.070, loss_actor:568.866\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 76/200 [03:53<06:21,  3.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 74    204 steps,total reward:-56.48  \n",
      "episode: 74：loss critic:0.050, loss_actor:561.323\n",
      "episode 75     14 steps,total reward:-0.80   \n",
      "episode: 75：loss critic:0.035, loss_actor:563.250\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 38%|███▊      | 77/200 [03:55<06:15,  3.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 76    153 steps,total reward:-67.98  \n",
      "episode: 76：loss critic:0.056, loss_actor:562.359\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|███▉      | 79/200 [03:55<06:00,  2.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 77     22 steps,total reward:-3.10   \n",
      "episode: 77：loss critic:0.043, loss_actor:557.420\n",
      "episode 78     16 steps,total reward:-1.20   \n",
      "episode: 78：loss critic:0.059, loss_actor:562.043\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 40%|████      | 80/200 [03:55<05:53,  2.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 79     24 steps,total reward:-5.23   \n",
      "episode: 79：loss critic:0.043, loss_actor:552.517\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 40%|████      | 81/200 [03:56<05:47,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 80     69 steps,total reward:-13.73  \n",
      "episode: 80：loss critic:0.047, loss_actor:560.811\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 41%|████      | 82/200 [03:56<05:40,  2.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 81     20 steps,total reward:-2.86   \n",
      "episode: 81：loss critic:0.031, loss_actor:575.607\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 42%|████▏     | 83/200 [03:57<05:34,  2.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 82     64 steps,total reward:-20.61  \n",
      "episode: 82：loss critic:0.047, loss_actor:557.194\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42%|████▎     | 85/200 [03:58<05:22,  2.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 83     37 steps,total reward:-14.41  \n",
      "episode: 83：loss critic:0.034, loss_actor:554.230\n",
      "episode 84     11 steps,total reward:-3.54   \n",
      "episode: 84：loss critic:0.062, loss_actor:558.540\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 43%|████▎     | 86/200 [03:58<05:15,  2.77s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 85     30 steps,total reward:-9.54   \n",
      "episode: 85：loss critic:0.050, loss_actor:573.273\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 44%|████▎     | 87/200 [03:58<05:09,  2.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 86     27 steps,total reward:-5.80   \n",
      "episode: 86：loss critic:0.045, loss_actor:560.396\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 44%|████▍     | 88/200 [03:58<05:04,  2.71s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 87     22 steps,total reward:-3.07   \n",
      "episode: 87：loss critic:0.048, loss_actor:551.505\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 44%|████▍     | 89/200 [03:59<04:58,  2.69s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 88     57 steps,total reward:-25.98  \n",
      "episode: 88：loss critic:0.049, loss_actor:563.984\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 45%|████▌     | 90/200 [03:59<04:53,  2.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 89     35 steps,total reward:-12.02  \n",
      "episode: 89：loss critic:0.056, loss_actor:553.661\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 46%|████▌     | 91/200 [04:00<04:48,  2.64s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 90     57 steps,total reward:-29.10  \n",
      "episode: 90：loss critic:0.047, loss_actor:560.492\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 46%|████▌     | 92/200 [04:00<04:42,  2.62s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 91     22 steps,total reward:-4.47   \n",
      "episode: 91：loss critic:0.030, loss_actor:554.787\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 46%|████▋     | 93/200 [04:01<04:37,  2.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 92     32 steps,total reward:-12.72  \n",
      "episode: 92：loss critic:0.031, loss_actor:555.051\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 47%|████▋     | 94/200 [04:02<04:33,  2.58s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 93    159 steps,total reward:-41.53  \n",
      "episode: 93：loss critic:0.046, loss_actor:564.866\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 48%|████▊     | 95/200 [04:03<04:29,  2.57s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 94    100 steps,total reward:-44.51  \n",
      "episode: 94：loss critic:0.044, loss_actor:565.639\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 48%|████▊     | 96/200 [04:04<04:25,  2.55s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 95     80 steps,total reward:-15.95  \n",
      "episode: 95：loss critic:0.046, loss_actor:565.533\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 48%|████▊     | 97/200 [04:05<04:20,  2.53s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 96     41 steps,total reward:-18.19  \n",
      "episode: 96：loss critic:0.050, loss_actor:586.646\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 49%|████▉     | 98/200 [04:05<04:16,  2.51s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 97     75 steps,total reward:-29.63  \n",
      "episode: 97：loss critic:0.041, loss_actor:564.888\n",
      "episode 98      4 steps,total reward:-0.02   \n",
      "episode: 98：loss critic:0.030, loss_actor:569.432\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 50%|█████     | 100/200 [04:06<04:06,  2.46s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 99     30 steps,total reward:-8.50   \n",
      "episode: 99：loss critic:0.056, loss_actor:564.356\n",
      "Models saved successfully\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 50%|█████     | 101/200 [04:07<04:02,  2.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 100    78 steps,total reward:-40.19  \n",
      "episode:100：loss critic:0.040, loss_actor:570.407\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 51%|█████     | 102/200 [04:08<03:58,  2.43s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 101    95 steps,total reward:-34.93  \n",
      "episode:101：loss critic:0.048, loss_actor:564.798\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 52%|█████▏    | 103/200 [04:08<03:54,  2.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 102    27 steps,total reward:-5.66   \n",
      "episode:102：loss critic:0.035, loss_actor:560.269\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 52%|█████▏    | 104/200 [04:09<03:50,  2.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 103    58 steps,total reward:-7.07   \n",
      "episode:103：loss critic:0.043, loss_actor:554.595\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 52%|█████▎    | 105/200 [04:10<03:46,  2.39s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 104   113 steps,total reward:-22.56  \n",
      "episode:104：loss critic:0.046, loss_actor:563.412\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 53%|█████▎    | 106/200 [04:13<03:44,  2.39s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 105   294 steps,total reward:-46.42  \n",
      "episode:105：loss critic:0.048, loss_actor:565.234\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 54%|█████▎    | 107/200 [04:14<03:41,  2.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 106    64 steps,total reward:-19.71  \n",
      "episode:106：loss critic:0.046, loss_actor:581.322\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 54%|█████▍    | 108/200 [04:14<03:37,  2.36s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 107    49 steps,total reward:-29.63  \n",
      "episode:107：loss critic:0.038, loss_actor:567.370\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 55%|█████▍    | 109/200 [04:15<03:33,  2.34s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 108    48 steps,total reward:-10.69  \n",
      "episode:108：loss critic:0.040, loss_actor:573.515\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 55%|█████▌    | 110/200 [04:15<03:29,  2.33s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 109    38 steps,total reward:-14.83  \n",
      "episode:109：loss critic:0.054, loss_actor:576.145\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56%|█████▌    | 112/200 [04:16<03:21,  2.29s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 110    63 steps,total reward:-31.89  \n",
      "episode:110：loss critic:0.047, loss_actor:572.165\n",
      "episode 111    16 steps,total reward:-1.11   \n",
      "episode:111：loss critic:0.042, loss_actor:589.562\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 56%|█████▋    | 113/200 [04:17<03:18,  2.28s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 112    30 steps,total reward:-10.72  \n",
      "episode:112：loss critic:0.044, loss_actor:574.853\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 57%|█████▋    | 114/200 [04:18<03:14,  2.26s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 113    52 steps,total reward:-12.60  \n",
      "episode:113：loss critic:0.040, loss_actor:565.035\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 57%|█████▊    | 115/200 [04:18<03:11,  2.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 114    26 steps,total reward:-6.27   \n",
      "episode:114：loss critic:0.043, loss_actor:573.317\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 58%|█████▊    | 116/200 [04:20<03:08,  2.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 115   126 steps,total reward:-13.23  \n",
      "episode:115：loss critic:0.037, loss_actor:566.393\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 58%|█████▊    | 117/200 [04:20<03:04,  2.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 116    25 steps,total reward:-3.74   \n",
      "episode:116：loss critic:0.053, loss_actor:590.572\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 59%|█████▉    | 118/200 [04:21<03:01,  2.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 117    47 steps,total reward:-15.19  \n",
      "episode:117：loss critic:0.046, loss_actor:549.647\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 60%|█████▉    | 119/200 [04:22<02:58,  2.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 118    35 steps,total reward:-12.96  \n",
      "episode:118：loss critic:0.052, loss_actor:578.317\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 60%|██████    | 120/200 [04:22<02:54,  2.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 119    27 steps,total reward:-6.38   \n",
      "episode:119：loss critic:0.033, loss_actor:571.811\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 60%|██████    | 121/200 [04:24<02:52,  2.18s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 120   158 steps,total reward:-36.33  \n",
      "episode:120：loss critic:0.046, loss_actor:570.362\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 61%|██████    | 122/200 [04:24<02:49,  2.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 121    43 steps,total reward:-14.66  \n",
      "episode:121：loss critic:0.063, loss_actor:574.107\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 62%|██████▏   | 123/200 [04:25<02:46,  2.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 122    86 steps,total reward:-13.76  \n",
      "episode:122：loss critic:0.054, loss_actor:580.696\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 62%|██████▏   | 124/200 [04:26<02:43,  2.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 123    59 steps,total reward:-17.86  \n",
      "episode:123：loss critic:0.062, loss_actor:574.937\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 62%|██████▎   | 125/200 [04:26<02:40,  2.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 124    31 steps,total reward:-13.17  \n",
      "episode:124：loss critic:0.040, loss_actor:566.101\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 63%|██████▎   | 126/200 [04:27<02:36,  2.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 125    22 steps,total reward:-3.84   \n",
      "episode:125：loss critic:0.045, loss_actor:567.005\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 64%|██████▎   | 127/200 [04:28<02:34,  2.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 126   163 steps,total reward:-42.00  \n",
      "episode:126：loss critic:0.053, loss_actor:569.459\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 64%|██████▍   | 128/200 [04:29<02:31,  2.11s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 127    96 steps,total reward:-29.93  \n",
      "episode:127：loss critic:0.043, loss_actor:572.772\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 64%|██████▍   | 129/200 [04:30<02:28,  2.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 128    48 steps,total reward:-25.60  \n",
      "episode:128：loss critic:0.036, loss_actor:572.750\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 65%|██████▌   | 130/200 [04:31<02:25,  2.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 129    59 steps,total reward:-25.58  \n",
      "episode:129：loss critic:0.059, loss_actor:585.129\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 66%|██████▌   | 131/200 [04:31<02:22,  2.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 130    20 steps,total reward:-2.02   \n",
      "episode:130：loss critic:0.073, loss_actor:571.383\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 66%|██████▌   | 132/200 [04:32<02:20,  2.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 131   119 steps,total reward:-15.05  \n",
      "episode:131：loss critic:0.050, loss_actor:571.558\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 66%|██████▋   | 133/200 [04:33<02:17,  2.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 132    45 steps,total reward:-12.75  \n",
      "episode:132：loss critic:0.035, loss_actor:572.886\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 67%|██████▋   | 134/200 [04:33<02:14,  2.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 133    43 steps,total reward:-5.78   \n",
      "episode:133：loss critic:0.037, loss_actor:578.611\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 68%|██████▊   | 135/200 [04:35<02:12,  2.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 134   149 steps,total reward:-36.63  \n",
      "episode:134：loss critic:0.045, loss_actor:575.719\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 68%|██████▊   | 136/200 [04:36<02:10,  2.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 135   105 steps,total reward:-23.08  \n",
      "episode:135：loss critic:0.052, loss_actor:576.502\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 68%|██████▊   | 137/200 [04:36<02:07,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 136    26 steps,total reward:-5.06   \n",
      "episode:136：loss critic:0.041, loss_actor:577.574\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 69%|██████▉   | 138/200 [04:38<02:05,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 137   187 steps,total reward:-30.07  \n",
      "episode:137：loss critic:0.050, loss_actor:576.485\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 70%|██████▉   | 139/200 [04:38<02:02,  2.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 138    41 steps,total reward:-21.18  \n",
      "episode:138：loss critic:0.044, loss_actor:572.741\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 70%|███████   | 140/200 [04:40<02:00,  2.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 139   189 steps,total reward:-24.91  \n",
      "episode:139：loss critic:0.046, loss_actor:569.704\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 70%|███████   | 141/200 [04:41<01:57,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 140    69 steps,total reward:-31.29  \n",
      "episode:140：loss critic:0.042, loss_actor:582.274\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 71%|███████   | 142/200 [04:42<01:55,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 141    29 steps,total reward:-5.29   \n",
      "episode:141：loss critic:0.045, loss_actor:581.057\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 72%|███████▏  | 143/200 [04:42<01:52,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 142    76 steps,total reward:-35.58  \n",
      "episode:142：loss critic:0.043, loss_actor:573.292\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 72%|███████▏  | 144/200 [04:43<01:50,  1.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 143    31 steps,total reward:-8.63   \n",
      "episode:143：loss critic:0.051, loss_actor:582.174\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 72%|███████▎  | 145/200 [04:43<01:47,  1.96s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 144    54 steps,total reward:-14.35  \n",
      "episode:144：loss critic:0.044, loss_actor:582.078\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 73%|███████▎  | 146/200 [04:46<01:46,  1.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 145   293 steps,total reward:-36.86  \n",
      "episode:145：loss critic:0.056, loss_actor:578.840\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 74%|███████▎  | 147/200 [04:47<01:43,  1.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 146    24 steps,total reward:-6.23   \n",
      "episode:146：loss critic:0.039, loss_actor:561.307\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 74%|███████▍  | 148/200 [04:47<01:41,  1.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 147    31 steps,total reward:-10.81  \n",
      "episode:147：loss critic:0.049, loss_actor:574.294\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 74%|███████▍  | 149/200 [04:48<01:38,  1.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 148   102 steps,total reward:-16.92  \n",
      "episode:148：loss critic:0.051, loss_actor:586.048\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 75%|███████▌  | 150/200 [04:49<01:36,  1.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 149    56 steps,total reward:-19.99  \n",
      "episode:149：loss critic:0.041, loss_actor:577.617\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 76%|███████▌  | 151/200 [04:50<01:34,  1.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 150    96 steps,total reward:-15.31  \n",
      "episode:150：loss critic:0.051, loss_actor:576.848\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 76%|███████▌  | 152/200 [04:54<01:32,  1.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 151   308 steps,total reward:-42.37  \n",
      "episode:151：loss critic:0.044, loss_actor:578.970\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 76%|███████▋  | 153/200 [04:54<01:30,  1.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 152    44 steps,total reward:-23.89  \n",
      "episode:152：loss critic:0.041, loss_actor:583.484\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 77%|███████▋  | 154/200 [04:55<01:28,  1.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 153    81 steps,total reward:-11.18  \n",
      "episode:153：loss critic:0.042, loss_actor:579.098\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 78%|███████▊  | 155/200 [04:56<01:26,  1.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 154    71 steps,total reward:-15.92  \n",
      "episode:154：loss critic:0.045, loss_actor:595.183\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 78%|███████▊  | 156/200 [04:57<01:24,  1.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 155    98 steps,total reward:-29.61  \n",
      "episode:155：loss critic:0.053, loss_actor:587.398\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 78%|███████▊  | 157/200 [04:59<01:21,  1.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 156    99 steps,total reward:-29.66  \n",
      "episode:156：loss critic:0.038, loss_actor:581.344\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 79%|███████▉  | 158/200 [05:00<01:19,  1.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 157   102 steps,total reward:-18.64  \n",
      "episode:157：loss critic:0.041, loss_actor:580.537\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 80%|███████▉  | 159/200 [05:00<01:17,  1.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 158    32 steps,total reward:-11.50  \n",
      "episode:158：loss critic:0.056, loss_actor:592.714\n",
      "episode 159     5 steps,total reward:-0.03   \n",
      "episode:159：loss critic:0.121, loss_actor:585.257\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 80%|████████  | 161/200 [05:03<01:13,  1.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 160   279 steps,total reward:-56.52  \n",
      "episode:160：loss critic:0.050, loss_actor:584.917\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 81%|████████  | 162/200 [05:04<01:11,  1.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 161    33 steps,total reward:-9.94   \n",
      "episode:161：loss critic:0.049, loss_actor:586.163\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 82%|████████▏ | 163/200 [05:05<01:09,  1.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 162    89 steps,total reward:-13.02  \n",
      "episode:162：loss critic:0.052, loss_actor:582.779\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|████████▎ | 165/200 [05:05<01:04,  1.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 163    22 steps,total reward:-3.91   \n",
      "episode:163：loss critic:0.054, loss_actor:592.434\n",
      "episode 164    17 steps,total reward:-3.16   \n",
      "episode:164：loss critic:0.056, loss_actor:566.038\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 83%|████████▎ | 166/200 [05:06<01:02,  1.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 165    36 steps,total reward:-14.27  \n",
      "episode:165：loss critic:0.047, loss_actor:572.787\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 84%|████████▎ | 167/200 [05:06<01:00,  1.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 166    47 steps,total reward:-13.09  \n",
      "episode:166：loss critic:0.058, loss_actor:588.366\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 84%|████████▍ | 168/200 [05:08<00:58,  1.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 167   134 steps,total reward:-15.50  \n",
      "episode:167：loss critic:0.049, loss_actor:574.048\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 84%|████████▍ | 169/200 [05:08<00:56,  1.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 168    40 steps,total reward:-15.31  \n",
      "episode:168：loss critic:0.047, loss_actor:584.880\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 85%|████████▌ | 170/200 [05:09<00:54,  1.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 169    44 steps,total reward:-32.34  \n",
      "episode:169：loss critic:0.061, loss_actor:576.524\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 86%|████████▌ | 171/200 [05:10<00:52,  1.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 170   168 steps,total reward:-24.80  \n",
      "episode:170：loss critic:0.043, loss_actor:582.283\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 86%|████████▌ | 172/200 [05:12<00:50,  1.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 171   151 steps,total reward:-18.24  \n",
      "episode:171：loss critic:0.057, loss_actor:589.798\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 86%|████████▋ | 173/200 [05:13<00:48,  1.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 172    22 steps,total reward:-5.08   \n",
      "episode:172：loss critic:0.077, loss_actor:586.642\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 87%|████████▋ | 174/200 [05:13<00:46,  1.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 173    32 steps,total reward:-1.21   \n",
      "episode:173：loss critic:0.058, loss_actor:590.302\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 88%|████████▊ | 175/200 [05:15<00:45,  1.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 174   168 steps,total reward:-26.86  \n",
      "episode:174：loss critic:0.045, loss_actor:586.230\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 88%|████████▊ | 176/200 [05:16<00:43,  1.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 175    24 steps,total reward:-2.54   \n",
      "episode:175：loss critic:0.040, loss_actor:593.785\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 88%|████████▊ | 177/200 [05:17<00:41,  1.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 176    74 steps,total reward:-28.72  \n",
      "episode:176：loss critic:0.059, loss_actor:590.411\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 89%|████████▉ | 178/200 [05:17<00:39,  1.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 177    39 steps,total reward:-14.10  \n",
      "episode:177：loss critic:0.038, loss_actor:589.525\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 90%|████████▉ | 179/200 [05:20<00:37,  1.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 178   276 steps,total reward:-66.43  \n",
      "episode:178：loss critic:0.054, loss_actor:579.259\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 90%|█████████ | 180/200 [05:21<00:35,  1.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 179    66 steps,total reward:-22.70  \n",
      "episode:179：loss critic:0.041, loss_actor:594.377\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 90%|█████████ | 181/200 [05:21<00:33,  1.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 180    21 steps,total reward:-3.27   \n",
      "episode:180：loss critic:0.061, loss_actor:569.034\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 91%|█████████ | 182/200 [05:23<00:31,  1.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 181   122 steps,total reward:-21.13  \n",
      "episode:181：loss critic:0.051, loss_actor:589.067\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 92%|█████████▏| 183/200 [05:25<00:30,  1.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 182   146 steps,total reward:-8.86   \n",
      "episode:182：loss critic:0.050, loss_actor:584.994\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 92%|█████████▏| 184/200 [05:25<00:28,  1.77s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 183    38 steps,total reward:-17.72  \n",
      "episode:183：loss critic:0.056, loss_actor:589.972\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 92%|█████████▎| 185/200 [05:25<00:26,  1.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 184    38 steps,total reward:-15.24  \n",
      "episode:184：loss critic:0.054, loss_actor:571.132\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 93%|█████████▎| 186/200 [05:26<00:24,  1.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 185    42 steps,total reward:-17.66  \n",
      "episode:185：loss critic:0.072, loss_actor:596.464\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 94%|█████████▎| 187/200 [05:26<00:22,  1.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 186    21 steps,total reward:-0.62   \n",
      "episode:186：loss critic:0.073, loss_actor:576.983\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 94%|█████████▍| 188/200 [05:26<00:20,  1.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 187    31 steps,total reward:-5.94   \n",
      "episode:187：loss critic:0.048, loss_actor:575.805\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 94%|█████████▍| 189/200 [05:27<00:19,  1.73s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 188    33 steps,total reward:-7.48   \n",
      "episode:188：loss critic:0.066, loss_actor:584.551\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 95%|█████████▌| 190/200 [05:27<00:17,  1.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 189    35 steps,total reward:-12.74  \n",
      "episode:189：loss critic:0.037, loss_actor:583.410\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 96%|█████████▌| 191/200 [05:28<00:15,  1.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 190    26 steps,total reward:-1.86   \n",
      "episode:190：loss critic:0.067, loss_actor:572.289\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 96%|█████████▌| 192/200 [05:28<00:13,  1.71s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 191    45 steps,total reward:-20.58  \n",
      "episode:191：loss critic:0.051, loss_actor:576.762\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 96%|█████████▋| 193/200 [05:28<00:11,  1.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 192    23 steps,total reward:-3.40   \n",
      "episode:192：loss critic:0.064, loss_actor:573.724\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|█████████▊| 195/200 [05:29<00:08,  1.69s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 193    71 steps,total reward:-11.66  \n",
      "episode:193：loss critic:0.063, loss_actor:582.597\n",
      "episode 194    14 steps,total reward:-0.05   \n",
      "episode:194：loss critic:0.045, loss_actor:594.169\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 98%|█████████▊| 196/200 [05:30<00:06,  1.68s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 195    31 steps,total reward:-2.73   \n",
      "episode:195：loss critic:0.074, loss_actor:565.209\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 98%|█████████▊| 197/200 [05:30<00:05,  1.68s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 196    24 steps,total reward:-4.80   \n",
      "episode:196：loss critic:0.043, loss_actor:569.673\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 99%|█████████▉| 198/200 [05:30<00:03,  1.67s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 197    39 steps,total reward:-13.38  \n",
      "episode:197：loss critic:0.060, loss_actor:564.922\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "100%|█████████▉| 199/200 [05:31<00:01,  1.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 198    23 steps,total reward:-2.74   \n",
      "episode:198：loss critic:0.053, loss_actor:577.467\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "100%|██████████| 200/200 [05:31<00:00,  1.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 199    34 steps,total reward:-14.47  \n",
      "episode:199：loss critic:0.079, loss_actor:563.580\n",
      "Models saved successfully\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "data = agent.learning(max_episode_num = 200, display = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No handles with labels found to put in legend.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAEKCAYAAADenhiQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAIABJREFUeJzsvXtcXGed+P/+zAADCdAEEiDJpAVSSBNiiQRLadPES+3NS211tV0vdXW9fNXv6td1V+utdbXrbVdXf+u6utpd77XrZdvV1tpqk5g2xQYa2pA2JA1pMiSBBkiAAAMz8/z+mDMww3UOYWaeNp/36zWvMM/zzDnvHOacD89djDEoiqIoykLiybSAoiiK8uJDg4uiKIqy4GhwURRFURYcDS6KoijKgqPBRVEURVlwNLgoiqIoC44GF0VRFGXB0eCiKIqiLDgaXBRFUZQFJyvTApli2bJlpry8PNMaiqIoLyiam5tPGmOWz1XunA0u5eXl7N6929VnnnnmGS666KIUGZ0dtrqplzts9QJ73dTLHWfrJSLPJVNOm8VccOLEiUwrzIitburlDlu9wF439XJHurw0uCiKoigLjgYXRVEUZcE5Z/tcFEVRznXGxsYIBAKMjIxMycvNzcXv95OdnT2vY2twcYHNo8tsdVMvd9jqBfa6qZc74r0CgQAFBQWUl5cjIuPpxhh6enoIBAJUVFTM6zwvmmYxEblGRPaLyEER+UQqzlFUVJSKwy4Itrqplzts9QJ73dTLHfFeIyMjFBcXJwQWABGhuLh42hpNsrwogouIeIFvAdcC64GbRWT9Qp+nq6troQ+5YNjqpl7usNUL7HVTL3dM9pocWOZKT5YXS7PYJcBBY8whABG5C7ge2LeQJ/H7/UmVOxMMMTIWpjjfN55mjBn/ZZ0JhsjN9uL1zPzL6zw1zOIcL0sW5QAwMDLG44d76eoPsnJJHluqlvHosz2EIoat1cvH3Ywx7Dvez/FTI2yuWkZutnf8mIPBEGeCIUoLcxPScrwecrIS/844NTTKWNiwvMDHaCjCk4FTHOge5LI1xVxQvDjh89v2d9NYWUxxvo8H2k6wtrSA8mWLp71m8ddhaDTEopzEr2D/yBi/ffI4gyMhypct5sp1JfSPhOg4eYb1KwqneM5FJGLYceB51q0opLQwl1A4QpbXw8pVq7jvqeNcUbWMgtxsHn32JO0nBjBAWWEu61YUsnRxDn/u6GXN8sVULs8H4LmeM+w71s/qokVkez30DAbpOTPKhSX5rFtRmHDuE6dHONI7RM3KQhb7Zr7VjDHc23qMUNiwcdWyhLzOU8M813OGJXk5rFtRMO0Nb4xhNBzBlxX9XYcjhvueOk5hXjYb/UtY7PNy7NQIp4ZHWbM8n85TwwT6hqgvL6IwN5tgKMzhk0OsXJJLQe707etj4QjPmwKyTp6hYtniKfmRiGFbezfFi31c7D8PgEDfMCNjYapKC8bLPddzhsW+LJbF3Rvx/4+nOk+zfkUhWV4PfWdGyc32kpcT/X+NjIXZf2KAsvNyWZ7vw+PcP9Pdl5FIdPt2zwz32OmhMR7e301BbhavWlcaTRseY1GOl/0nBnj2+UFCYcNFKwpYv6IwqQdte9cAbcdOcyYY5vUbV1KyYiW/23ucyy9cNu117Tw1zNHeIS4pLyJiDKeHxxKeGQAHuwfZ3v48ZYW5LC/wsdjnxb90EV39I7QdO83W6hIW5XjZeeAkK5fkYTDsPHASESjMzea8vGwuWlFIYW4Wf+7oxb90EWtWrZrz/7IQvFiCyyrgaNz7ANAwuZCI3A7cBlBSUsK2bdsS8svKyqioqKC9vZ2amhp27NiRkD84OMirX/1qOjo68Pv9dHV1cfTo0YQyq1at4o4/9fDos738vyurWEuA00HD5x8bYas/i9rlXr7aMkaOB/5ik583Vudw/NgxIsbwH08GWVvk5a2NFbzj7sOYSJhff+Aybv3Zo+wMhAiZifMsL/Dx/EAQj8BnXlFG1sBxnjydw6PHQnQORgsW+LwsyhZGI0I4FGJg1OAR+MLleeRmC/d3FfDHZ7p4w8ZVvGtDDj/f3YnXA1v82Xz18WEOnjbccskqfru3i87+seg1BC5bmcU7anLYdTzCLw6EGAyG+L+vvJBXlwX54F1HWbnYw+2X5eIRYXR0lLq6Op7v6+e/njjFPa3H+UxjHkNjhi/9eYQb1+bx1Xe+kubmZg6ZEm6/5ynOjE38R2uKPQTOCKdHwuRmCZvOP49Cc4ZTwQivWJ1N9VIvX2sZpWGtn6tLh3is/zxCp7vZuDTqe1/HKHfvj/6cm+UhYgyfvKqCvfuf5ReHhC3+LC5dkcVXHp+5+i9A1VIPg+FsjvUHZyzTuDqXY0MeRoJBrq1dzU8fO0wwDF6B6yqzueHCbC6+uJZvbz/Etmf7ycvxsCInSM+woaU7DESbEj5/wwbWep/niKeMj/+ilbFI9BwrFgvXVWTzwddewve3H+DEkGFVYRZ3txznxBlDYQ68rCyL/nAWj3fO3ZyR5YHFOVkMBENEDCzxCW9Zm8Mjx0IcGTCIx0NBNvh8Po70nGEsAlkCH67z4S/w4Fm+hmeOdJG7OJ8H206wt2sYgEVZMBqBkOP9xvWFvPaSar7x2yfY83yYXC+8tjKbwTFYVlLKIjNMTXkZP33sME1Hz7CuyMOaJV5+eyj6e1u/ooDXXpjLL/cPcbD7TNRdoDhPKMoVgmHIys7m9tdvYLDvJD9+vJPmrhDBECxfJLyiqojL167g6PFuDgxk8Uj7cbrOGAzgEfjspbk0n1nK/z55fNrrlOsFI4LX4yHbA3k5WaxeFOLylVn0jRieG4hwoC9C52Bk/DPf+P3TZBHi+JCwLE94aYmXp095+OirKtmw1PDjvYP85M+dhA2ULhKGQoaBUVi1JA9PZAy82UgkxJHTY7P+DnO9kO2FgdE5f93jFOXCT9+5kUWeMJFIhIGBgYR8ESE/P58zZ6LXuqWlhf7+/uRPEDuOMWbuUpYjIm8CrjHG/LXz/u1AgzHmQzN9pr6+3ridob9t2zZe/vKXz1rmYPcgV35tO/6leQT6hnnflkoGgyF+0nQEYLw2clFZAX94pps7btjAWxsu4BfNAT72362sOC+Xf35zLX/5H00AFOZm0T8S4uZLzuf1tSs5v3gRjx48yf8+eZxXrl3Or/ccY9+x04TC0Ztl0wVLueGlq/AvzeN3e08wGo6wKMdLKGwoLczluzsOcXVNKX1DY/y5o5dlBTl4RNj+d6/g0n/8A6PhCA99dCsvu+Mh8n1ZnB4eo2LZYv72qmqqSwv4RXOA//jTIVael0fnqWGuqFrGU52nefW6Um69bh11n38QgPdcUcGeo6dYm3eGL9zyat72vSYeefYkArx7cwWnh8e4e3cAgHddXsHmqmLe88NmNl2wlE9dt47K5Yv52Z+P8E8PtLNx9RJubljNniOn2HnwJM/1DGGA19eu5Is3voR1n/0dxsAb6/z8siWAL8vDg/9vK2ORCNd+409ctqaYSyqKODkwyp6jfezt7AcTISvLy8hYmFVL84hE4NcfuAyPRzhxeoQnA6d5fiDIy8qX8sizJ/nTgZOcX7SITRcs5aXnL+X4qWHCxlC82MeSRdn8/PGj/LTpCJsuWMpIKMwTR05xRdUy3nbpBdz/1HH+Z88xLizJJ9vr4enj/VxSUYRHoOXIKcIRw63XXsQVVcv5ux8/wpMnw+RlexkeC9NQUcTfvKqKzr5hftz0HE8GTlO8OIeeM6PkZnsYGYuwtrSAazaUcejkGX639zgRA59+zTrWLM9n/4kBzoyGKCvMZcmiHA52D1BSkMuqpXn86cBJhkZDLMnLxr90Ef/68EGO9A6xZFE2171kBVke4fmBIMbA+cWL8J7uZHu3j/auAUKRxOfG0kXZfOLaixCE1sAp8n1ZnF+8iL2dp/nZn4+Ol3l7Yzm7D/fy6LM95Hg9+LI8DARDAPiyPPxFvZ+7Hw8wGo5w40tXcUHxYu7efZTOU8MsWZTNrddexGgoQuDUMIG+YY6dGmb0TD+nIz6OnRomYmBRjper1pdSdl4ebcdO88jBk8R0C3OzaKgsZsPK83hZ+VI+/PM9jIyGGQiGuPmS1SwvyGXN8sXUrCzEI0Lzc320HesnJ8tDJGIIhiKcCYZ48OkuBkZC48fcsOo8rtlQxmVriukbGuPjv3ySvv4hPnzVOr7/SAfHT42wZFEOY+EIr7qohF890clb6ldTX76U/94doPS8XGpWFvJU52nCYUNutofBYIialefxF/V++odD9A2N0j88xpHeIQpys6kqzednTUcYGg3zlktWc2polHAEtlYvJy/Hy+nhMXoHR9l77DS9Z0ZpqCji0Mkz/Oaxffzgg1chIjz99NNcdNFFM9aIn3nmGdatW5eQLiLNxpj6uZ6XL5bg0gjcboy52nl/K4Ax5oszfWa+weXrbdmsWbaYr71l47RlPv0/T3H37gCPfuKVfP3Bdn7SdASvR7jpZas5NTTGo8+e5O73NXJhST5v/V4TTwZO87U31/LJX+9lLBzh9PAYVSX5BPqG+eirq/ni/U9z2+tquOWy8mnPd3IwyGf+Zy+5Iz185IbLE5qspuOL9z/Nd7YfAuCzr13P8FiYrz6wn4c+uoUrvxatqd30stXc9fhRfvl/GhkNGV56/pKE5rUH2k7wkbv28PraldxxwwZu/PajLFmUwyevu4hr/uVPLMrxMjQa/Ut8dYGH33z0Sjb+w+/5wMvXsP/EIK2BU4yMhnn1+lLyc7P44a7oahJrli/mfz54eUITwmAwxOIc75SRLLf85+P0DAb56ptque6bfyLH62E0HKGhooi9nae5sCSfnjOjDIyEePCjWygpiDYFnhoa5fpvPUL36SF+/aErePO/76J/JMS3/rKO11y8YtZrNxfxTX4nB4MUL84Zf/+L5gD/80Rn9CH2stW85WWrERGGRkOcCYZZXhBtDvnDHx/mgPd8uvuDXFRWwI11q8jyRpsCIxHD3buP8l+PHua9Wyp5fe1KjvQOUV68eLz5p7t/JPodimuKSpZTQ6P85snjvPbiFePNsfFs27aNi192GV/47T7WLM/niqpl+JcuIjfbQ47XM+45+Zr88ZluQhHDy9cux5flJRIxHDp5Bv/SPHxZHvqGxniu5wylhbmsXJLHvmP9dA+M8PK1JQAMj4b5RUuArVXLOb940bRemy69nH96YD/nLcrhXZeXJ/g/PxCk89QwWR5hbVkB2XGev287wXt/1Mz1G1fyL2/ZmHQ/w2AwxBNH+igvXox/ad6Uz4Ujhoe3bePKV76C0VCEkVCYnsFRXvPNPzE0GuY9V1TwqdcseLdwUsT/kdzR0UFBQcGUTv3YaLGBgYEpo8WSDS4vlmaxx4EqEakAOoGbgL9MxYmGR0MMj4WnzesfGeOXzZ1cX7uSZfk+PvPa9TQ/18dzPUN8+Moqluf7CIYi4w/qL914Mdd+Ywfv/VEzXo9w9/su5T0/bOZA9yCvq13Je7ZU8tZLz5/SLxHPsnwf337bJrZt2zZnYAF435Y1/HjXc1Quz+eWy8rZefAkAD9tmmjeu+vxoxQvzmHj6qXT9gtdXVNG621Xjfd/lBTkEugbostpMrrjhg10nBxicCTEnY908OC+LoyBK6qWs2HleTz0dLRD8cY6P5urlnH9xpX8pOkIH3rFhVPapvOn6asQEapL8vnRoR72d0Wr6//05lp2tD/PJ69bx69aAnzht0+zbkUhX3/LxvHAArBkUQ73fPByHty2k4vKCvnKmy7m8cN9XPeSsjmv3VzE35yT+xTetMnPmzZN7RtYlJOV8Pv1eoT3b10z7fE9HuGmS87npkvOH0+L9QXFKCnMpSSuT80NSxbl8LZLL5i1TNHiHL725un/sJoOERnv04jh8QgXlkx4Fy3OoWjxRDBYv7KQ9Uz0X+XleHn7HF4Fudl87voN0+YtL/CNB+/JXFVTxu8+cgVrlue76sDO92VxRdXMazd6PUKWc+/kZEX7NAtzs/n/bn4pu5/r42NXrU36XKnE7/cTCAR4/vnnp+TF5rnMlxdFcDHGhETkQ8ADgBe40xjTttDnERE8IkRmqO0d6RlieCw8fjPlZnv5+fsa6RkMjj/g4msA5xcv4o8feznP9QxRVpjL+cWLeMPGVdz5SAevc/6Kni2wTHZLhqLFOfzv/91Mcb4Pr0fYsDJ6E/+yJYDXI7xi7XIeerqbl68tmXXAQXzHekmhj5YjfXT1R9v56y8o4oaX+nn8cC93PtLBtx4+SE6Wh42rlzgdjVnkZntpXFMMwKYLith0gbthm9VlBQRDEf7wdDcegatrSnl97Uog2uzWuKaYdWWF03boLlmUQ8ni6O/hmg0ruGbD2dVYFpKzHaGTSmx1O1uvi8oK5y40D6bzetW60inBNt3Ee2VnZ897HstcvCiCC4Ax5j7gvlSeY+vWrXxpz5+IzNCSOBaOduj54h685+VFR2zMRGlhbsLorfdvrSQ/N4tXXFTi2i1Z4v/aLc73sfK8XI6dHmHdikLeWOfnoae7uXJd8ucvLcil98wogb5oh27sr8Ra/xLysr10nDzDpZVF44H1y2+8mGyvZ9bgNRfVTrPPH5/pprx48fhIKYjePDUrz5v1826uVzqx1QvsdVMvd6TL60UxzyVdtLa24pFoe+R0xDo5s7zzf2iWFOby0VdXJ7QLJ+s2X2pWRR/Etf7zuLqmjB+86xKurkm+mai0MBpM9h07zXl52eNBJCfLw9ri6N8vl1YWj5e/9iUruHL92f31VuU0qwyNhhOaWJLlbK5XKrHVC+x1Uy93pMtLg4sLampqnGax6fNjNZcsT/ova01Nzbw/+xInuFzsX4LHI2ytXj7j/IDpKHGCy1OdpymZ1Lb9qpdE+wfig8tCsNiXxaoleQBUlboPLmdzvVKJrV5gr5t6uSNdXhpcXNDe3o5HmLHPJexEneyzqLnMl/b29nl/9rI1xWR7hUsr57dcRaw/qas/mNDEB1C/NMjHr7mIl5Uv/FIYa8uiTWNVJe5HRp3N9UoltnqBvW7q5Y50eWlwcUF3dzcyS80lFI41i6X/snZ3d8/7s/XlRTx529VTRh4lS3xAidViYgT7e/g/L19zVv0rMxGrscynWexsrlcqsdUL7HVTL3eky+tF06GfLmbrc5loFrNzVM1sxJbYmA/Fi3PweoRwxCQM+001V9eU8fTxgXk1iymKklq05uKS2YYiL0SH/gsRj0dY7szrKC2cfj5BKqg7fyk/fNclCSPFFEWxAw0uLvGIEIlMn5fJDv1MEwsq6ay5KIpiL+feU/AsqKysRGbp0I/1uWSiQ7+ysjLt54wnNit8cs0l014zoV7usdVNvdyRLi8NLi4oKCjAI8JMy7GFI5nr0C8ocD9iaiGJBZXJo8Uy7TUT6uUeW93Uyx3p8tLg4oLe3l48nplrLmNOe1l2Bjr0e3t7037OeMqLF5Ob7ZmyhlOmvWZCvdxjq5t6uSNdXjpazAWlpaV45OSczWKpGHY7F6WlmV2v6G2XXsCr1pUmrJ0GmfeaCfVyj61u6uWOdHlpzcUFu3fvnnWey3iHfgaaxdxuH7DQ5GZ7p92hMNNeM6Fe7rHVTb3ckS4vDS4uSWZtsUx06CuKotiEBheXzLa22HiH/jk4FFlRFCUefQq6ZLa1xV7IM/QVRVEWEg0uLsjKyppzbTGP4GpF4YUiK8vOsRnq5Q5bvcBeN/VyR7q8ZKb+gxc79fX1Zj4dW+//UTOHe87wu49smZL3xfuf5j8fOUz7F65dCEVFURTrEJFmY0z9XOW05uKClpaWKfNcDp88Q/0XHuK5njOEwiYjc1xibjaiXu6w1QvsdVMvd6TLS4OLCzZu3DilWezxw72cHAxyuGeIcMRkZBhyzM1G1MsdtnqBvW7q5Y50eWlwcUFbW9uUVZGfff4MAGOhCGPhSMaGIbe1tWXkvHOhXu6w1QvsdVMvd6TLS4OLC3p6epx5LhNpzz4/CERHioXCJiOz82NuNqJe7rDVC+x1Uy93pMtLg4tLptZcosFlNBxhLBLROS6KoihocHFN/JL7Y+EIR3qGABgNRWsuOjtfURRFg4tr4jcLe65naHzJl7GwIRSJZKxDX1EUxSb0SeiCqqqqhLXFYk1iAKOhMKGwydjs/Kqqqoycdy7Uyx22eoG9burljnR5aXBxgc/nS1hbLD64RGsuhuwM1Vx8vvTtXe8G9XKHrV5gr5t6uSNdXhpcXDA4OOjMc3FqLt1nWLIoG3A69MORjI0WGxwcnLtQBlAvd9jqBfa6qZc70uWlwcUFRUVFzsKV0fdHes9QXRLdMjTTHfpFRUUZOe9cqJc7bPUCe93Uyx3p8tLg4oKWlhY8IuN9LqNhQ26Ol2yvROe5ZHAo8rm+1IRb1Ms9trqplzt0+RdLiV9y3xiDVyDb63GCiyFLhyIriqJocHFL/Npi4YjBI0K21xPXLKaXVFEURZ+ELomfoR8x0b1bcrI8jIZNRjv0FUVRbCIjwUVE/kJE2kQkIiL1k/JuFZGDIrJfRK6OS7/GSTsoIp+IS68QkSYn/ecikpMq7+hQ5Im1xSKR6OZgObGaSyRzHfrn+rBHt6iXe2x1Uy93vNiHIu8FbgR2xCeKyHrgJqAGuAb4NxHxiogX+BZwLbAeuNkpC/Bl4OvGmAuBPuDdqZJubGzE44mvucSaxZwO/XDmOvQbGxszct65UC932OoF9rqplzvS5ZWRJ6Ex5mljzP5psq4H7jLGBI0xHcBB4BLnddAYc8gYMwrcBVwvIgK8EviF8/kfAG9IlXdzc3PC2mIRY8abxTLdod/c3JyR886FernDVi+w10293JEuL9s2eV4FPBb3PuCkARydlN4AFAOnjDGhacpPQURuB24DKCkpYdu2bQn5ZWVlVFRU0N7eTk1NDTt27JhyDBPOJRw2DA4OMhIc5WR3N8Fhw7GuYc4MRRgaHGB4eJiOjg6qq6vZuXPnlGNs3ryZ9vZ2KioqCAQCdHZ2JuSvXr2a0tJSAoEAFRUV7Nq1a8oxtmzZQltbG9XV1XR0dDAwMJDw/ykvL6eoqIiuri78fj9NTU2TrwVbt26ltbWVmpoa2tvb6e7uTihTWVlJQUEBvb29lJaWMnlb6KysLDZv3kxLSwsbN26kra1tynLeVVVVnDx5ksHBQYqKiqYMg/T5fDQ2NtLc3MymTZtobW2lr68voczatWvxer2MjIxQUFBAa2trQn5eXh4NDQ3jx2hpaaG/vz+hzPr16wmHw0QiEXw+35TrlZ+fT319/fgxdu/ePWWy2YYNGwgGg3g8HrxeL/v27UvILywspK6ubvwYTU1NDA8PJ5Spra1lYGCA3NxcwuEw+/cn/o21dOlSgPFj7Nq1i2AwmFCmrq6O3t5e8vPzCQaDHDhwICG/uLiYmpoa9uzZQ11dHTt37iQUCiWUqa+vp6uri6KiIgYGBjh06FBCfklJCdXV1bS1tVFbW8v27dsxxiRcs4aGBgKBAKWlpfT29nL48OGEYyRzPzU2NtLR0YHf76erq4ujR48m5K9atQq/3z/n/bRv374FvZ9OnDiRkD+f+2nyd2yh7iefz3fW91N3d/dZ3U/JICZ+c5IFREQeAsqmyfqUMeYep8w24GPGmN3O+38FHjPG/Nh5/33gfudz1xhj/tpJfzvR4HK7U/5CJ301cL8xZsNcfvX19WbyL3cuWltb+f0JH9/ZfoiD/3gdW7/6MC9dvYTDPUMU5Gbx9PF+rq4p444bXuLquAtBa2srtbW1aT/vXKiXO2z1Anvd1MsdZ+slIs3GmPq5yqWs5mKMuXIeH+sEVse99ztpzJDeAywRkSyn9hJffsHp6+vDIyum9LnEmsXGMrhw5eS/TmxBvdxhqxfY66Ze7kiXl21Dke8FbhIRn4hUAFXAn4HHgSpnZFgO0U7/e0202vUw8Cbn87cA96RSMH6eSyQSfT8+WiysS+4riqJA5oYi3yAiAaAR+K2IPABgjGkD7gb2Ab8DPmiMCTu1kg8BDwBPA3c7ZQE+DnxURA4S7YP5firdYxUTYwwRY/B6cEaLGcZ0hr6iKAqQoQ59Y8yvgV/PkHcHcMc06fcB902TfojoaLK04JFo8IiYiRn6sWaxcMSQrdscK4qiWNcsZjVr164dr7lEjCFios1i2V4PwVA0uGSq5rJ27dqMnHcu1MsdtnqBvW7q5Y50eWlwcYHX60XGay4munClJzpDf2g0OtQzUx36Xq83I+edC/Vyh61eYK+berkjXV4aXFwwMjIy3ixmDITjRosNjYYBMtahPzIykpHzzoV6ucNWL7DXTb3ckS4vDS4uKCgoSGwWi1sVeTgWXDJUcykoKMjIeedCvdxhqxfY66Ze7kiXlwYXF7S2tiZ06BvDeHAJOeOTM7Xk/uRZtragXu6w1QvsdVMvd6TLy7blX6xH4mou0WaxxKYwHYqsKIqSRM1ForxNRD7rvD9fRNI29Nc2xvtcIjjzXIScuICSqWYxRVEUm0imDeffiE52vNl5P0B0+ftzksQ+l4mhyDEyteS+oiiKTSTTLNZgjKkTkScAjDF9qdyQy2by8vLwRCaGIkecZrGcrMw3i+Xl5WXkvHOhXu6w1QvsdVMvd6TLK5k/s8eczboMgIgsByIptbKUhoaGuHkuE81i8TWXTHXoNzQ0ZOS8c6Fe7rDVC+x1Uy93pMsrmSfhN4ku1VIiIncAO4F/TKmVpTQ3N08/Qz++5pKhPpdzfWMit6iXe2x1Uy93WLNZmDHmJyLSDLwKEOANxpinU25mIZs2baL9z0eA6LpiAF4RfBbUXDZt2pSR886FernDVi+w10293JEurxmfhCJSFHsB3cDPgJ8CXU7aOUdLS8t4zSUUjgYXj0B21kRtxZuhmsvkHelsQb3cYasX2OumXu5Il9dsNZdmov0sApwP9Dk/LwGOABUpt7OM/v5+JH85AKFItNvJ4xFy4tbqyVSH/ny2IU0H6uUOW73AXjf1cke6vGasuRhjKowxlcBDwOuMMcuMMcXAa4Hfp8XOQmLzXGLNYtEZ+hMBJVPNYoqiKDaRzJPwUmcvFQCMMfcDl6VOyW5irV5jCc1ime/QVxRFsYlk5rkcE5FhKMNcAAAgAElEQVRPAz923r8VOJY6JbuZruZiQ4e+oiiKTSTzJLwZWE50OPKvgRImZuufU6xfv358bbH4Ppf4mkumOvTXr1+fkfPOhXq5w1YvsNdNvdyRLq9khiL3Ah8WkYLoWzOYei07CYfD4zWXUCSuWSyh5pKZ4BIOhzNy3rlQL3fY6gX2uqmXO9LllczClS9xln7ZC7SJSLOIbEi9mn1EIpGJ4OL0uUQXrsz82mKRiJ2LJqiXO2z1Anvd1Msd6fJK5kn4HeCjxpgLjDEXAH8LfDe1Wnbi8/km5rk4vyARISdunkumhiL7fL6MnHcu1MsdtnqBvW7q5Y50eSUTXBYbYx6OvTHGbAMWp8zIYvbu3Tu+ttjMzWKZqbns3bs3I+edC/Vyh61eYK+berkjXV7JjBY7JCKfAX7kvH8bcCh1SnYTq7mEwxPLv+ToUGRFUZQEkvkz+11ER4v9ynktc9LOSSY69CPj73U/F0VRlESSGS3WB/wNgLP0/mJjjJ3rGqSBWOwYbxabtOS+bnOsKIqS3Gixn4pIoYgsBp4C9onI36VezT7y8/PH+1zCcX0uPgs2C8vPz8/IeedCvdxhqxfY66Ze7kiXVzJtOOudmsobgPuJLlj59pRaWUp9ff14s9jE8i+TNgvLULNYfX19Rs47F+rlDlu9wF439XJHurySeRJmi0g20eByrzFmDGdXynON+M3CwnEz9L0ewSPRWoxHNwtLQL3cYasX2OumXu5Il1ey81wOEx1+vENELgDOyT6XTZs2TVNzieblZHky2pl/rm9M5Bb1co+tburljoxvFhbDGPNNY8wqY8x1JspzwCvS4GYdu3fvHl9bLH7hSojOb8lkZ/7u3bszdu7ZUC932OoF9rqplzvS5TXjaDEReZsx5sci8tEZinwtRU7WMjg4SG7R5EmU0fc5Xg9jZG65h8FBO5d8Uy932OoF9rqplzvS5TXbUOTYLPyCdIi8UJhYWyw2zyWaHj+RUlEU5VxnxuBijPmO8+/nFvqkIvJV4HXAKPAs8FfGmFNO3q3Au4Ew8DfGmAec9GuAbwBe4HvGmC856RXAXUAx0a2Z326MGV1o5xgTa4tNLFwJ0WaxiDknxzkoiqJMIZl5LpUi8r8i8ryIdIvIPSJSeZbnfRDYYIy5GGgHbnXOtR64CagBrgH+TUS8zuTNbwHXAuuBm52yAF8Gvm6MuRDoIxqYUsbUeS6x4CI6O19RFMUhmafhT4G7gRXASuC/gZ+dzUmNMb83xoSct48Bfufn64G7jDFBY0wHcBC4xHkdNMYccmoldwHXS/RJ/0rgF87nf0B0yHRK2LBhw0TNJRxbFTn6PifLm9EO/Q0b7NwFQb3cYasX2OumXu5Il1cyC1cuMsb8KO79jxd4hv67gJ87P68iGmxiBJw0gKOT0huINoWdigtU8eWnICK3A7cBlJSUsG3btoT8srIyKioqaG9vp6amhh07diTkj42NsWRNLQBDI0EA9j71FBz3MjI0zGjIcODAAfx+Px0dHVRXV7Nz584pHps3b6a9vZ2KigoCgQCdnZ0J+atXr6a0tJRAIEBFRQW7du2acowtW7bQ1tZGdXU1HR0dHD16lOzs7PH88vJyioqK6Orqwu/309TUNPlasHXrVlpbW6mpqaG9vZ3u7u6EMpWVlRQUFNDb20tpaemUUSZZWVls3ryZlpYWNm7cSFtbGz09PQlliouLgWgnYlFRES0tLQn5Pp+PxsZGmpub2bRpE62trfT19SWUWbt2LV6vl5GREQoKCmhtbU3Iz8vLo6GhYfwYLS0t9PcnjpZfv3494XCYSCSCz+fjiSeeSLhe+fn51NfXjx9j9+7dUzo+N2zYQDAYxOPx4PV62bdvX0J+YWEhdXV148doampieHg4oUxtbS0DAwPk5uYSDofZv39/Qn5OTg7Lli0bP8auXbsIBoMJZerq6ujt7SU/P59gMMiBAwemXPOamhr27NlDXV0dO3fuJBQKJZSpr6+nq6uLoqIiBgYGOHQocS3akpISqquraWtro7a2lu3btzM6OppwzRoaGggEApSWltLb28vhw4cTjjHX/QTQ2NhIR0cHfr+frq4ujh49mpC/atWqOe+nyspK9u3bt6D304kTJxLy53M/dXZ2Jlyvhbifqqqq8Pl8Z3U/LV++nEgkclb3UzKImaOfQES+TLS56S6ikyffAiwFvgrjO1VO97mHgLJpsj5ljLnHKfMpoB640RhjRORfgceMMT928r9PdFUAgGuMMX/tpL+daHC53Sl/oZO+GrjfGDNnaK6vrzduh+QdP36ck+FFvO5fd/LeLZV8d8chfvLXDVx+4TLe9O1HGQyG+N1Htrg65kJx/PhxVqxYkZFzz4Z6ucNWL7DXTb3ccbZeItJsjJlzmn8yNZc3O/++b1L6TUSDzbT9L8aYK+cQfCfwWuBVZiLCdQKr44r5nTRmSO8BlohIllN7iS+/4Hi9XsQZbRzbiVLiJ1GGMtcs5vV6M3bu2VAvd9jqBfa6qZc70uWVzCTKille8+rYd0Z+/T3wemPMUFzWvcBNIuJzRoFVAX8GHgeqRKRCRHKIBrZ7naD0MPAm5/O3APfMxykZ9u3bN96BH1v+xeu89y/NY9WSvFSdOik3G1Evd9jqBfa6qZc70uU1Z81FRBYBHwXON8a8V0SqgLXGmN+cxXn/FfABDzqjrx4zxrzfGNMmIncD+4AQ8EFjTNjx+BDwANGhyHcaY9qcY30cuEtEvgA8AXz/LLzmJDYgbCxuyX2Af7zhJak8raIoyguKZJrF/pPo/JHLnPedREeMzTu4xPpIZsi7A7hjmvT7gPumST9EdDRZWhivuYQThyJnZWh7Y0VRFBtJ5om4xhjzFWAMwGnGOmd3xIoNRR6LJM7QVxRFUSZIJriMikgezjL7IrIGCM7+kRcnhYWFM06izDSFhYWZVpgW9XKHrV5gr5t6uSNdXskEl9uA3wGrReQnwB+Idsafc9TV1U2sLTZp+ZdMU1dXl2mFaVEvd9jqBfa6qZc70uWVzGixB4EbgXcSnZlfb4zZllotO4nfLGzyDP1Mc65vTOQW9XKPrW7q5Y50ec05ifLFynwmUQIc7R3iiq88zJXrSnjo6W7u//AVrFthZ/VXURRloUl2EqUOcXJBU1PTeE3FtmaxyctR2IJ6ucNWL7DXTb3ckS6vGYOLM4lRiWN4eDhuP5fEbY4zzeQ1rGxBvdxhqxfY66Ze7kiX12w1l18AiMgf0mLyAmGiQz+S8F5RFEWZYLZJlB4R+SRQPd1Wx8aYc26bY4jbLCxs11BkRVEUm5it5nIT0d0gs4hudTz5dU4ik4Yia3BRFEWZymzbHO8HviwiTxpj7p+p3LlEbW0tESeWjE+itGRIRG1tbaYVpkW93GGrF9jrpl7uSJdXMo/GR0XkayKy23n9s4icl3IzCxkYGBivqYyF7epzGRgYyLTCtKiXO2z1Anvd1Msd6fJKJrjcCQwQ3dflzUA/0cUszzlyc3Pjlty3ayhybm5uphWmRb3cYasX2OumXu5Il1cyqyKvMca8Me7950RkT6qEbCYcDiNOOI4FF0sqLoTD4UwrTIt6ucNWL7DXTb3ckS6vZGouwyKyOfZGRC4H7BzAnWL2798/0Sxm2VDkyXux24J6ucNWL7DXTb3ckS6vZGou7wd+GNfP0kd0x8dzklgrWGw/F68lwUVRFMUm5gwuxphWoFZECp33/Sm3spiJmosORVYURZmJZGougAaVGDJpKLJYMhRZURTFJvTR6IKlS5fGrS0W7XOxpVls6dKlmVaYFvVyh61eYK+berkjXV665L5LwhHDmk/ex6IcL0OjYZ7+h2vIy/GmwFBRFMU+FmzJfRH5CxEpcH7+tIj8SkTs3GItxSRsFmbZDP1zfWMit6iXe2x1Uy93WLNZmLP8y8XOcOQvAF8FPmuMaUiHYKqYb80FoPwTv8UjEDFw4I5ryfZaEmEURVFSzEJuFhabcfMa4LvGmN8COWcj90Jl165dAOOBJfqzHX0uMTfbUC932OoF9rqplzvS5ZVMcOkUke8AbwHuExFfkp970REMBoHEgGLJ6i/jbrahXu6w1QvsdVMvd6TLK5kg8WbgAeBqY8wpoAj4u5RaWU4suIhMLMGvKIqiTDDjPBcRKYp7uy0uLQjMr7PiRUIsntjSJKYoimIbs02ibAYMIMD5RJd9EWAJcASoSLmdpcSCii1zXBRFUWxjxmYxY0yFMaYSeAh4nTFmmTGmGHgt8Pt0CdpEXV10BHasn8Wm2BJzsw31coetXmCvm3q5I11eyfS5XGqMuS/2xtmV8rLUKdlLb28vEFdzsaU3nwk321Avd9jqBfa6qZc70uWVzNpix0Tk08CPnfdvBY6lTsle8vPzATv7XGJutqFe7rDVC+x1Uy93pMsrmZrLzcBy4NfAr5yfb06llK2MD0X2TIwWs4VzfdijW9TLPba6qZc7rBiKLCJe4JPGmA8bY15qjKkzxnzEGGNnfS/FHDhwALCzWSzmZhvq5Q5bvcBeN/VyR7q8Zg0uxpgwsHm2MvNBRD4vIk+KyB4R+b2IrHTSRUS+KSIHnfy6uM/cIiIHnNctcembROQp5zPflDRMPPFY2CymKIpiE8k0iz0hIveKyNtF5MbY6yzP+1VjzMXGmI3Ab4DPOunXAlXO673At2F8fs1tQANwCXCbiMTWjf428J64z11zlm5zEotfFlVcFEVRrCKZDv1coAd4ZVyaIdr/Mi8mbTy22DkewPXAD010Nc3HRGSJiKwAXg48GGuOE5EHgWtEZBtQaIx5zEn/IfAG4P75uiWD1lwURVFmJ5ltjv8qFScWkTuAdwCngVc4yauAo3HFAk7abOmBadJnOuftRGtAlJSUsG3btoT8srIyKioqaG9vp6amhh07diTkDw8PEwwGCYdCAEQi4SnHWLVqFX6/n46ODqqrq9m5c+cUj82bN9Pe3k5FRQWBQIDOzs6E/NWrV1NaWkogEKCiomLahea2bNlCW1sb1dXVdHR0MDw8nOBSXl5OUVERXV1d+P1+mpqaJl8Ltm7dSmtrKzU1NbS3t9Pd3Z1QprKykoKCAnp7eyktLWXyKtJZWVls3ryZlpYWNm7cSFtbGz09PQllfD4fJ0+eZHBwkKKiIlpaWqbkNzY20tzczKZNm2htbaWvry+hzNq1a/F6vYyMjFBQUEBra2tCfl5eHg0NDePHaGlpob8/cePU9evXEw6HiUQi+Hy+KdcrPz+f+vr68WPs3r2bwcHBhGNs2LCBYDCIx+PB6/Wyb9++hPzCwkLq6urGj9HU1MTw8HBCmdraWgYGBsjNzSUcDrN///6E/Egkugld7Bi7du2a0gFbV1dHb28v+fn5BIPBKW3oxcXF1NTUsGfPHurq6ti5cych5zsbo76+nq6uLoqKihgYGODQoUMJ+SUlJVRXV9PW1kZtbS3bt2+fcs0aGhoIBAKUlpbS29vL4cOHE44x1/0E0NjYSEdHB36/n66uLo4ePZqQn8z9tHTpUvbt27eg99OJEycS8udzP02+XgtxP1VVVeHz+c7qfsrNzaW7u/us7qdkSGbJ/Vzg3UAN0VoMAMaYd83xuYeAsmmyPmWMuSeu3K1ArjHmNhH5DfAlY8xOJ+8PwMeJ1lxyjTFfcNI/AwwTXZbmS8aYK530K4CPG2NeO+t/ivktuR+JRPB4PFz+pT/SeWqYVUvyeOQTr5z7g2kg5mYb6uUOW73AXjf1csfZei3kkvs/Ihokrga2A35gYK4PGWOuNMZsmOZ1z6SiPwHe6PzcCayOy/M7abOl+6dJTwl79uwBJoYg29QqFnOzDfVyh61eYK+berkjXV7JBJcLjTGfAc4YY35AdF+Xs9ooTESq4t5eDzzj/Hwv8A5n1NilwGljzHGiqzJfJSJLnY78q4AHnLx+EbnUGSX2DmBy8FowJpZ/kYR/beBcX2rCLerlHlvd1MsdNi3/Mub8e0pENgDnASVned4vicheEXmSaKD4sJN+H3AIOAj8B/ABAKcj//PA487rH+Lm2nwA+J7zmWdJYWd+rL031qFv0zyX6dqibUC93GGrF9jrpl7uSJdXMqPFvuvUFj5DtGaR7/w8b4wxb5wh3QAfnCHvTuDOadJ3AxvOxidZYp2i8fu52MLkDltbUC932OoF9rqplzvS5ZXMaLHvOT9uBypTq/PCIBZUdMl9RVGU6ZkzuIjIs8BjwJ+APxlj2lJuZTk29rkoiqLYRDJ9LuuB7wDFwFdF5FkR+XVqtezGxmYxRVEUm0gmuISJduqHgQjQ7bzOOerro0O7xcIO/ZibbaiXO2z1Anvd1Msd6fJKJrj0A/8CdAC3GGMajTHvS62WnXR1dQF2NovF3GxDvdxhqxfY66Ze7kiXV7L7uewgOuT3LhH5nIi8KrVadlJUVARAbHKrRRWXcTfbUC932OoF9rqplzvS5TVncDHG3GOM+TvgfUTnobyT6ErG5xwDA9GFCcZrLhZFl5ibbaiXO2z1Anvd1Msd6fKaM7iIyC9F5CDwDWAR0VnwS2f/1IuT2OJ+YmGz2OSFB21BvdxhqxfY66Ze7kiXVzKTKL8IPOFsHKYQN0PfouCiKIpiE8n0uewDbhWR70J0XTARmXPV4RczOhRZURRldpIJLv8JjAKXOe87gS+kzOgFgG4WpiiKMjvJBJc1xpiv4CxgaYwZAs7Jp2pJSXS9zlifi03zXGJutqFe7rDVC+x1Uy93pMsrmeAyKiJ5OFsRi8gaIDj7R16cVFdXAxM1F5sqLjE321Avd9jqBfa6qZc70uWVTHC5DfgdsFpEfgL8Afj7lFpZSltbdFk1GydRxtxsQ73cYasX2OumXu5Il9es2xw7G3D5gSHgUqLNYY8ZY06mxS6FzGeb4xhv+14TOw+e5JUXlXDnO1+2wGaKoij2siDbHDv7q9xnjOkxxvzWGPObF0NgmS/bt28HJprDLOpyGXezDfVyh61eYK+berkjXV7JNIu1iIj+eQ7Eank2NovNVgPNJOrlDlu9wF439XJHurySmUTZALxVRJ4DzhBtGjPGmItTamYxOhRZURRldpIJLlen3OIFxsTaYhkWURRFsZRktjl+Lh0iLyRsXFtMURTFJvRvbxc0NDQAdjaLxdxsQ73cYasX2OumXu5Il5cGFxcEAgEgvkM/kzaJxNxsQ73cYasX2OumXu5Il5cGFxeUlpYCcZuFWRRdYm62oV7usNUL7HVTL3eky0uDiwt6e3sBO/tcYm62oV7usNUL7HVTL3eky0uDiwsOHz4MTAQVm/ZzibnZhnq5w1YvsNdNvdyRLi8NLvNgvENfr56iKMq06ONxHkxsFmZPzUVRFMUmNLjMg1hMsalZTFEUxSY0uLigrKwMsHMocszNNtTLHbZ6gb1u6uWOdHlpcHFBRUUFEN/nYk90ibnZhnq5w1YvsNdNvdyRLi8NLi5ob28H7FwVOeZmG+rlDlu9wF439XJHurw0uLigpqYGiJ/nkkmbRGJutqFe7rDVC+x1Uy93pMsro8FFRP5WRIyILHPei4h8U0QOisiTIlIXV/YWETngvG6JS98kIk85n/mmpHAI144dOwA7m8VibrahXu6w1QvsdVMvd6TLK2PBRURWA1cBR+KSrwWqnNd7gW87ZYuA24juLXMJcJuILHU+823gPXGfuybV7jY2iymKothEJmsuXwf+HojfFu164IcmymPAEhFZQXRPmQeNMb3GmD7gQeAaJ6/QGPOYsyXzD4E3pFp8YlXkVJ9JURTlhUkym4UtOCJyPdBpjGmd1Iq1Cjga9z7gpM2WHpgmfabz3k60BkRJSQnbtm1LyC8rK6OiooL29nZqamqmVB8HBwcJBoOcOnUKgNOnTk05xqpVq/D7/XR0dFBdXc3OnTuneGzevJn29nYqKioIBAJ0dnYm5K9evZrS0lICgQAVFRXs2rVryjG2bNlCW1sb1dXVdHR0MDg4mOBSXl5OUVERXV1d+P1+mpqaJl8Ltm7dSmtrKzU1NbS3t9Pd3Z1QprKykoKCAnp7eyktLWX37t0J+VlZWWzevJmWlhY2btxIW1sbPT09CWVGR0c5efIkg4ODFBUV0dLSkpDv8/lobGykubmZTZs20draSl9fX0KZtWvX4vV6GRkZoaCggNbW1oT8vLw8Ghoaxo/R0tJCf39/Qpn169cTDoeJRCL4fL4p1ys/P5/6+vrxY+zevZvBwcGEY2zYsIFgMIjH48Hr9bJv376E/MLCQurq6saP0dTUxPDwcEKZ2tpaBgYGyM3NJRwOs3///oT8WPnYMXbt2kUwGEwoU1dXR29vL/n5+QSDQQ4cOJCQX1xcTE1NDXv27KGuro6dO3cSCoUSytTX19PV1UVRUREDAwMcOnQoIb+kpITq6mra2tqora1l+/btU65ZQ0MDgUCA0tJSent7pywrMtf9BNDY2EhHRwd+v5+uri6OHj2akJ/M/WSMYd++fQt6P504cSIhfz730+TrtRD3U1VV1fj3d77309jYGN3d3Wd1PyWFMSYlL+AhYO80r+uBJuA8p9xhYJnz82+AzXHH+ANQD3wM+HRc+mectHrgobj0K4DfJOO3adMm45aHH37YGGPM5+5tMxd8/Dfma7/f7/oYqSLmZhvq5Q5bvYyx10293HG2XsBuk8QzNmU1F2PMldOli8hLgAogVmvxAy0icgnQCayOK+530jqBl09K3+ak+6cpnxIaGxuBieYwr0XtYjE321Avd9jqBfa6qZc70uWV9j4XY8xTxpgSY0y5MaacaFNWnTHmBHAv8A5n1NilwGljzHHgAeAqEVnqdORfBTzg5PWLyKXOKLF3APekyr2jowOYGCVmUWwZd7MN9XKHrV5gr5t6uSNdXhnpc5mF+4DrgIPAEPBXAMaYXhH5PPC4U+4fjDGxTQk+APwXkAfc77xSgt8frSTFuolsWrgy5mYb6uUOW73AXjf1cke6vDIeXJzaS+xnA3xwhnJ3AndOk74b2JAqv3i6urrIz8+f2M/FoqpLzM021MsdtnqBvW7q5Y50eekMfRfERrLYOBR58igbW1Avd9jqBfa6qZc70uWlwWUe6CRKRVGU2dHgMg9Eg4uiKMqsaHCZBzY2iymKotiEBhcXrFoVnfxvY4d+zM021MsdtnqBvW7q5Y50eWlwcUFsCF8sqOhQ5LlRL3fY6gX2uqmXO9LlpcHFBbHJRzLeLGZPcDnXJ2y5Rb3cY6uberkjXV4aXFxQXV0NxDeLZdImkZibbaiXO2z1Anvd1Msd6fKy6PFoP7EVWT0WztCfbrVYG1Avd9jqBfa6qZc70uWlwWUe6DwXRVGU2dHgMg/EwmYxRVEUm9DH4zzwWNihryiKYhMaXOaBNospiqLMjkQXIj73qK+vN5O3GZ2LUChEVlYWP9p1mM/c08a3/rKO11y8IjWCLom52YZ6ucNWL7DXTb3ccbZeItJsjKmfq5zWXFzQ3t4OxK8tlkmbRGJutqFe7rDVC+x1Uy93pMtLg4sLKioqgLhmMYuiS8zNNtTLHbZ6gb1u6uWOdHlpcHFBIBAA7OzQj7nZhnq5w1YvsNdNvdyRLi8NLi7o7OwE7JyhH3OzDfVyh61eYK+berkjXV4WPR5fOIiFM/QVRVFsQoPLPNChyIqiKLOjwWUeeJyr5tXgoiiKMi0aXFywevVqIL7mkkmbRGJutqFe7rDVC+x1Uy93pMtLg4sLSktLgYm+Fpv6XGJutqFe7rDVC+x1Uy93pMtLg4sLJg9Ftmmb43N92KNb1Ms9trqplzt0KLKFTJlEaU9sOecnbLlFvdxjq5t6uUMnUVrIrl27gLhJlBZFl5ibbaiXO2z1Anvd1Msd6fLS4DIPRIciK4qizIoGl3lgY7OYoiiKTWhwmQcNlUW8b2slF5UVZlpFURTFSuzbbOAFQGFuNrdeuy7TGoqiKNaim4W5IBKJ4PHYWdmz1U293GGrF9jrpl7uOFsv3SwsBbS1tWVaYUZsdVMvd9jqBfa6qZc70uWVkeAiIreLSKeI7HFe18Xl3SoiB0Vkv4hcHZd+jZN2UEQ+EZdeISJNTvrPRSQnVd7V1dWpOvRZY6ubernDVi+w10293JEur0zWXL5ujNnovO4DEJH1wE1ADXAN8G8i4hURL/At4FpgPXCzUxbgy86xLgT6gHenSrijoyNVhz5rbHVTL3fY6gX2uqmXO9LlZVuz2PXAXcaYoDGmAzgIXOK8DhpjDhljRoG7gOslOuHklcAvnM//AHhDquROnDiRqkOfNba6qZc7bPUCe93Uyx3p8spkcPmQiDwpIneKyFInbRVwNK5MwEmbKb0YOGWMCU1KVxRFUTJIyoYii8hDQNk0WZ8Cvg18HjDOv/8MvCtVLnFOtwO3AZSUlLBt27aE/LKyMioqKmhvb6empoYdO3Yk5A8ODhIMBuno6MDv99PV1cXRo0cTyqxatQq/309HRwfV1dXs3LlzisfmzZtpb2+noqKCQCAwZdvR1atXU1paSiAQoKKiYtrlGrZs2UJbWxvV1dV0dHQwODiY8P8pLy+nqKiIrq4u/H4/TU1Nk68FW7dupbW1lZqaGtrb2+nu7k4oU1lZSUFBAb29vZSWljJ5dF1WVhabN2+mpaWFjRs30tbWRk9PT0KZ0dFRTp48yeDgIEVFRbS0tCTk+3w+GhsbaW5uZtOmTbS2ttLX15dQZu3atXi9XkZGRigoKKC1tTUhPy8vj4aGhvFjtLS00N/fn1Bm/fr1hMNhIpEIPp9vyvXKz8+nvr5+/Bi7d+9mcHAw4RgbNmwgGAzi8Xjwer3s27cvIb+wsJC6urrxYzQ1NTE8PJxQpra2loGBAXJzcwmHw+zfvz8hP1Y+doxdu3YRDAYTytTV1dHb20t+fj7BYJADBw4k5BcXF1NTU8OePXuoq6tj586dhEKhhDL19fV0dXVRVFTEwMAAhw4dSsgvKSmhurqatrY2avBehm0AAAgpSURBVGtr2b59+5Rr1tDQQCAQoLS0lN7eXg4fPpxwjLnuJ4DGxsazvp+MMezbt29B76fJf93P536afL0W4n6qqqoa//7O934aGxuju7v7rO6npDDGZPQFlAN7nZ9vBW6Ny3sAaHReD8Sl3+q8BDgJZDnpCeVme23atMm45eGHH3b9mXRhq5t6ucNWL2PsdVMvd5ytF7DbJPGMzcgkShFZYYw57ry9Adjr/Hwv8FMR+RqwEqgC/kw0iFSJSAXQSbTT/y+NMUZEHgbeRLQf5hbgnmQcmpubT4rIcy7VVwLHXH4mXdjqpl7usNUL7HVTL3ecrdcFyRTKyCRKEfkRsJFos9hh4H2xYCMinyLaRBYCPmKMud9Jvw74F8AL3GmMucNJryQaWIqAJ4C3GWMS2xEWztsYY6xcUcxWN/Vyh61eYK+berkjXV7n7Az9+WDrlwXsdVMvd9jqBfa6qZc70uVl21BkRVEU5UWABhd3fC7TArNgq5t6ucNWL7DXTb3ckRYvbRZTFEVRFhytuSiKoigLjgaXJJlp4cwMeKwWkYdFZJ+ItInIh530GRcDTaPbYRF5yjn/bietSEQeFJEDzr9L5zpOCrzWxl2XPSLSLyIfycQ1c1ak6BaRvXFp014jifJN5zv3pIjUpdnrqyLyjHPuX4vIEie9XESG467bv6fZy/XCt2l0+3mc12ER2eOkp/OazfSMSO/3LJnJMOf6i+jw52eBSiAHaAXWZ8hlBVDn/FwAtBNdzPN24GMZvk6HgWWT0r4CfML5+RPAly34XZ4gOlY/7dcM2ALU4Uwcnu0aAdcB9xOd53Up0JRmr6uYmKD85Tiv8vhyGbhe0/7enPugFfABFc49602n26T8fwY+m4FrNtMzIq3fM625JMe0C2dmQsQYc9wY0+L8PAA8jd3rqV1PdEFRSPHCoknyKuBZY4zbCbQLgjFmB9A7KXmma3Q98EMT5TFgiYisSJeXMeb3ZmLdvscAfyrO7dZrFmZa+DbtbiIiwJuBn6Xq/DMxyzMird8zDS7JMdPCmRlFRMqBlwKxhY6mWww0nRjg9yLSLCLvddJKzcRqDCeA0gx4xXMTiTd8pq8ZzHyNbPrevYvoX7cxKkTkCRHZLiJXZMDHzcK3meAKoMsYE7/wW9qv2aRnRFq/ZxpcXqCISD7wS6KrGPQTXQx0DdGVD44TrZKnm83GmDqi++58UES2xGeaaB08Y8MTJbqR3OuB/3aSbLhmCWT6Gk2HRFfNCAE/cZKOA+cbY14KfJTokk2FaVSy7vc2DTeT+EdM2q/ZNM+IcdLxPdPgkhydwOq4934nLSOISDbRL81PjDG/AjDGdBljwsaYCPAfpLA5YCaMMZ3Ov93Arx2HrlgV2/m3e+YjpJxrgRZjTBfYcc0cZrpGGf/eicg7gdcCb3UeSDjNTj3Oz81E+zbStu3iLL+3jF8vABHJAm4Efh5LS/c1m+4ZQZq/ZxpckuNxnIUznb9+byK6yGbacdpyvw88bYz5Wlx6fBtp/GKg6fJaLCIFsZ+JdgbvJXqdbnGKJb2waIpI+Gsy09csjpmu0b3AO5zRPJcCp+OaNVKOiFwD/D3wemPMUFz6conuDhtb268KODT9UVLiNdPv7V7gJhHxSXSR29jCt+nmSuAZY0wglpDOazbTM4J0f8/SMXrhxfAiOqKinehfHJ/KoMdmotXZJ4E9zus64EfAU076vcCKNHtVEh2p0wq0xa4R0Q3d/gAcAB4CijJ03RYDPcB5cWlpv2ZEg9txYIxo2/a7Z7pGREfvfMv5zj0F1KfZ6yDRtvjY9+zfnbJvdH7He4AW4HVp9prx90Z0v6hngf3Aten+XTrp/wW8f1LZdF6zmZ4Raf2e6Qx9Rfn/27ufF5viMI7j7w8WClnILCg/yoIGo6amNNkYJllqaorZsBMSNUmRTFNolLDQLIRm2FsQapT8mDJKfuRPsGA1NWTBPBbf75g7l9zc+c5o6vOqW6fnnPM959y6PX3POfd5zKw43xYzM7PinFzMzKw4JxczMyvOycXMzIpzcjEzs+KcXMxmiaQeSTsKjDNW4nzMZpJfRTabYySNRcTi/30eZn/jmYvZNEjqkvQy9+jolzRf0pikS7mXxpCk5Xnbm5I68vL53G/jraSLObZG0uMcG5K0KsfXShpW6pXTW3X8bkkjeZ+zObZI0j1JbyS9l9Q5u9+KmZOLWd0kbQA6gdaI2AL8APaRqgG8iohG4Alwpmq/ZaSyJY0RsRmYSBhXgVs5dhu4kuOXgWsRsYn0j/CJcdpJZURaSEUcm3Ox0F3Ax4hoioiNwIPiF29Wg5OLWf3agGZgRKnjYBupDM44k0ULB0nlOCqNAt+A65L2ABN1u7YCd/LyQMV+rUzWRBuoGKc9f16TSoqsJyWbd8BOSRckbYuI0Wlep9k/W/C/T8BsDhNppnFySlA6XbXdlAebEfFdUgspGXUAh4HtNY71p4ejAs5FRP9vK1Kr2t1Ar6ShiOipMb5ZUZ65mNVvCOiQ1AC/epSvJv2uOvI2e4FnlTvlPhtLI+I+cAxoyqtekCpuQ7q99jQvP6+KT3gIHMjjIWmlpAZJK4CvETEI9JFa8ZrNKs9czOoUER8knSJ135xHqo57CPgCtOR1n0jPZSotAe5KWkiafRzP8SPADUndwGdgf44fJTWXOkFFy4KIeJSf+wynKuuMAV3AOqBP0ng+p4Nlr9ysNr+KbFaYXxU2820xMzObAZ65mJlZcZ65mJlZcU4uZmZWnJOLmZkV5+RiZmbFObmYmVlxTi5mZlbcTwKksfVgBlPaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learning_curve(data, 2, 1, #title=\"DDPGAgent performance on PuckWorld with continuous action space\", \n",
    "               x_name=\"episodes\", y_name=\"rewards of episode\")    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = agent.learning(max_episode_num = 100, display = True, \n",
    "                      explore = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
